{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp data.metadatasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metadatasets: a dataset of datasets\n",
    "\n",
    "> This functionality will allow you to create a dataset from data stores in multiple, smaller datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* I'd like to thank both Thomas Capelle (https://github.com/tcapelle)  and Xander Dunn (https://github.com/xanderdunn) for their contributions to make this code possible. \n",
    "* This functionality allows you to use multiple numpy arrays instead of a single one, which may be very useful in many practical settings. I've tested it with 10k+ datasets and it works well. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from tsai.imports import *\n",
    "from tsai.utils import *\n",
    "from tsai.data.validation import *\n",
    "from tsai.data.core import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class TSMetaDataset():\n",
    "    \" A dataset capable of indexing mutiple datasets at the same time\"\n",
    "    def __init__(self, dataset_list, **kwargs):\n",
    "        if not is_listy(dataset_list): dataset_list = [dataset_list]\n",
    "        self.datasets = dataset_list\n",
    "        self.split = kwargs['split'] if 'split' in kwargs else None            \n",
    "        self.mapping = self._mapping()\n",
    "        if hasattr(dataset_list[0], 'loss_func'): \n",
    "            self.loss_func =  dataset_list[0].loss_func\n",
    "        else: \n",
    "            self.loss_func = None\n",
    "\n",
    "    def __len__(self):\n",
    "        if self.split is not None: \n",
    "            return len(self.split)\n",
    "        else:\n",
    "            return sum([len(ds) for ds in self.datasets])\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if self.split is not None: idx = self.split[idx]\n",
    "        idx = listify(idx)\n",
    "        idxs = self.mapping[idx]\n",
    "        idxs = idxs[idxs[:, 0].argsort()]\n",
    "        self.mapping_idxs = idxs\n",
    "        ds = np.unique(idxs[:, 0])\n",
    "        b = [self.datasets[d][idxs[idxs[:, 0] == d, 1]] for d in ds]\n",
    "        output = tuple(map(torch.cat, zip(*b)))\n",
    "        return output\n",
    "\n",
    "    def _mapping(self):\n",
    "        lengths = [len(ds) for ds in self.datasets]\n",
    "        idx_pairs = np.zeros((np.sum(lengths), 2)).astype(np.int32)\n",
    "        start = 0\n",
    "        for i,length in enumerate(lengths):\n",
    "            if i > 0: \n",
    "                idx_pairs[start:start+length, 0] = i\n",
    "            idx_pairs[start:start+length, 1] = np.arange(length)\n",
    "            start += length\n",
    "        return idx_pairs\n",
    "    \n",
    "    @property\n",
    "    def vars(self):\n",
    "        s = self.datasets[0][0][0] if not isinstance(self.datasets[0][0][0], tuple) else self.datasets[0][0][0][0]\n",
    "        return s.shape[-2]\n",
    "    @property\n",
    "    def len(self): \n",
    "        s = self.datasets[0][0][0] if not isinstance(self.datasets[0][0][0], tuple) else self.datasets[0][0][0][0]\n",
    "        return s.shape[-1]\n",
    "\n",
    "\n",
    "class TSMetaDatasets(FilteredBase):\n",
    "    def __init__(self, metadataset, splits):\n",
    "        store_attr()\n",
    "        self.mapping = metadataset.mapping\n",
    "    def subset(self, i):\n",
    "        return type(self.metadataset)(self.metadataset.datasets, split=self.splits[i])\n",
    "    @property\n",
    "    def train(self): \n",
    "        return self.subset(0)\n",
    "    @property\n",
    "    def valid(self): \n",
    "        return self.subset(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create 3 datasets. In this case they will have different sizes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(#105) [(TSTensor(vars:5, len:50, device=cpu), TensorCategory(9)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(7)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(5)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(9)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(10)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2))] ...],\n",
       " (#134) [(TSTensor(vars:5, len:50, device=cpu), TensorCategory(1)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(2)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(7)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(6)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(10)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(4)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(9)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(8)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(3))] ...],\n",
       " (#143) [(TSTensor(vars:5, len:50, device=cpu), TensorCategory(5)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(10)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(6)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(5)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(10)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(8)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(7)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(1)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(6)), (TSTensor(vars:5, len:50, device=cpu), TensorCategory(7))] ...]]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = L(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'])\n",
    "dsets = []\n",
    "for i in range(3):\n",
    "    size = np.random.randint(50, 150)\n",
    "    X = torch.rand(size, 5, 50)\n",
    "    y = vocab[torch.randint(0, 10, (size,))]\n",
    "    tfms = [None, TSClassification(add_na=True)]\n",
    "    dset = TSDatasets(X, y, tfms=tfms)\n",
    "    dsets.append(dset)\n",
    "dsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<__main__.TSMetaDataset at 0x7fd7ffe75f28>, 5, 50)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadataset = TSMetaDataset(dsets)\n",
    "metadataset, metadataset.vars, metadataset.len"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll apply splits now to create train and valid metadatasets: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABAYAAABKCAYAAAAoj1bdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQrklEQVR4nO3dfXBV9Z3H8c8nCfKwWDQSQYgWF6wGUxMaC9qliHSXqltrqeITah+kdp12tlu7SrvbUYt01t3pbp1d2059oLBrS3VEXXXYVbci+DClCkuqIgi1dMASE5ogIkVzyXf/uCfd22tuwsNNbu6979dMJvec87u/8z1++V3nfvM7v+OIEAAAAAAAKE8VhQ4AAAAAAAAUDoUBAAAAAADKGIUBAAAAAADKGIUBAAAAAADKGIUBAAAAAADKGIUBAAAAAADKGIUBAEDRsv2U7fnJ63m2Hz+MvibYDttVyfZ/2f5MnuL8qO1NGdtbbf95PvpO+nvZ9sx89QcAAMoLhQEAQEHZnm77Odtv2m63/aztDx9sPxHx44iYndFv2J50qHFFxLkRsbSvdgdynoh4OiJOPtRYss63xPairP5PjYin8tE/AAAoP1WFDgAAUL5sv0/So5KulXSfpCMkfVTSO4WMK59sV0VEqtBxAAAA5MKMAQBAIX1AkiJiWUTsj4jfR8TjEfFLSbL92WQGwe3JjIKNtj/WU0dJ22eS16uT3c2299i+pIf2lba/Y3un7dck/WXW8czbFCbZXpXEsNP2vbnOY3um7e22F9hukfSj7n1ZIXzY9gbbHbZ/ZHtY9nVkxBJJDNdImifphuR8jyTH/3Brgu2htm+z/dvk5zbbQ5Nj3bF9zXar7R22P9dnlgAAQEmjMAAAKKRXJe23vdT2ubaP7qHNNEm/kjRa0k2SHrBd3VunETEjedkQESMj4t4emn1B0ickTZF0uqSLeunyFkmPSzpaUq2kf+vjPGMlVUt6v6RrcvQ5T9LHJU1UukDyzd6uKTnfHZJ+LOmfkvOd30Ozv5d0hqRGSQ2Spmb1PVbSKEnjJV0t6Xs5/rsDAIAyQWEAAFAwEbFb0nRJIelOSW22H7Y9JqNZq6TbIqIz+eK9SVl/3T9EFyf9bouIdkn/0EvbTqW/5I+LiH0R8UwvbSWpS9JNEfFORPw+R5vbM879bUmXHewF5DBP0sKIaI2INknfknRlxvHO5HhnRKyQtEdSXtY/AAAAxYnCAACgoCLilYj4bETUSqqXNE7SbRlNXo+IyNj+TdLmcI2TtC2r31xukGRJv0ieAPD5Pvpui4h9fbTJPnc+rklJP5nXkt3377LWPNgraWSezg0AAIoQhQEAwKARERslLVG6QNBtvG1nbJ8g6bd5ON0OScdn9ZsrrpaI+EJEjJP0RUnf7+NJBNHLsW7Z5+6+prcljeg+YHvsQfb9W6VnN/TUNwAAwHtQGAAAFIztU5KF8GqT7eOVnlL/84xmx0r6a9tDbM+VVCdpxQF0/4akP+3l+H1Jv7XJPfZf7yXOud0xSupQ+st51wGeJ5cvJeeuVnpdgO71CZolnWq7MVmQ8Oas9/V1vmWSvmm7xvZoSTdKuucQ4gMAAGWCwgAAoJDeUnpxwTW231a6IPCSpK9ltFkj6SRJO5W+F/+iiPjdAfR9s6SltnfZvriH43dKekzpL+LrJD3QS18fTmLcI+lhSV+JiNcO8Dy5/ETpBQ1fU3pxxUWSFBGvSloo6X8kbZaUvZ7B3ZImJ+d7qId+F0l6QdIvJb2YXNuig4gLAACUGf/xbZsAAAwetj8raX5ETC90LAAAAKWKGQMAAAAAAJQxCgMAAAAAAJQxbiUAAAAAAKCMMWMAAAAAAIAyRmEAAAAAAIAyVtUfndqjQ5rQH10DAAAARWtE3SuFDgE4bHtf2bszImoKHQfyp18KA+miwAv90zUAAABQpE65p6nQIQCHbV3Tut8UOgbkF7cSAAAAAABQxigMAAAAAABQxigMAAAAAABQxvppjQEAAAAAAAavtWvXHltVVXWXpHqV9h/NuyS9lEql5jc1NbX21IDCAAAAAACg7FRVVd01duzYupqamo6KiooodDz9paury21tbZNbWlrukvTJntqUclUEAAAAAIBc6mtqanaXclFAkioqKqKmpuZNpWdG9NxmAOMBAAAAAGCwqCj1okC35Dpzfv/nVgIAAAAAAAZYS0tL5cyZM0+WpJ07dw6pqKiI6urqlCStX7/+lWHDhuUsWqxevXrE4sWLj1myZMm2fMTSZ2HA9mJJn5DUGhE5px4AAAAAAFCsbDXls78Ire3t+NixY/dv3LhxgyRdd91140aOHLl/4cKFb3Qf7+zs1JAhQ3p874wZM/bOmDFjb75iPZBbCZZIOidfJwQAAAAAAO914YUXTrj88stPOO2000659tpra1euXDmisbHxlLq6uslTpkw5pbm5eagkPfroo0eeffbZk6R0UWHu3LkTpk6denJtbe0HFy1adOzBnrfPGQMRsdr2hIO+IgAAAAAAcFB27NhxxLp16zZWVVWpvb294vnnn984ZMgQPfTQQ0fecMMNtY899tivst+zZcuWYc8999ymXbt2VdbV1dVff/31bUOHDj3g9RPytsaA7WskXZPeOiFf3QIAAAAAUDY+/elPd1RVpb+qt7e3V15yySUnbt26dZjt6OzsdE/vmT179q7hw4fH8OHDU9XV1Z3bt2+vmjhxYueBnjNvTyWIiDsi4vSIOF2qyVe3AAAAAACUjZEjR3Z1v16wYMH4s846663Nmze//Mgjj2x59913e/wOnzk7oLKyUqlUqscCQi48rhAAAAAAgEFo9+7dlbW1te9K0g9/+MPR/XUeCgMAAAAAAAxCCxYsaLn55ptr6+rqJqdSqX47jyN6X4/A9jJJMyWNlvSGpJsi4u7e33N6SC/kK0YAAACgJHxobV6fhgYUxLqmdWvTt5AXt+bm5q0NDQ07Cx3HQGlubh7d0NAwoadjB/JUgsvyHhEAAAAAABgUuJUAAAAAAIAyRmEAAAAAAIAyRmEAAAAAAIAyRmEAAAAAAIAyRmEAAAAAAIAyRmEAAAAAAIABNm3atA8sX778fZn7Fi5ceOy8efNO6Kn91KlTT169evUISTrrrLMm7dy5szK7zXXXXTfuxhtvHHOwsfT5uEIAAAAAAEpd07qmpnz2t/ZDa9f2dnzu3Lnty5Ytq77wwgt3d+9bvnx59a233rq9r75XrVq1JR8xdmPGAAAAAAAAA+zKK6/sePLJJ0ft27fPkrRp06YjWltbh9xzzz3V9fX1dZMmTTr1q1/96rie3jt+/PgP7tixo0qSFixYMHbChAn1TU1NJ2/evHnoocTSTzMG1u6RvKl/+kYBjJa0s9BBIK/Iaekhp6WHnJYeclp6Djqn6/L690j0A8bpgXl/oQMoBWPGjNnf0NDw9v333z/qiiuu2LV06dLq888/v+OWW27ZMWbMmP2pVEof+chHTl6zZs3wadOm/b6nPp5++ukRDz74YPWLL764obOzU42NjZOnTJmy92Bj6a9bCTZFxOn91DcGmO0XyGdpIaelh5yWHnJaeshp6SGnpYecYqBdfPHF7ffee+/RV1xxxa4HHnig+s4779y6dOnS6iVLloxOpVJua2sb0tzcPCxXYWDlypUjzzvvvF1HHnlklyTNnj1716HEwa0EAAAAAAAUwOWXX77r2Weffd8zzzwzYt++fRU1NTWp22+/fcyqVateffXVVzfMmjXrzX379vX793YKAwAAAAAAFMCoUaO6zjzzzLfmz58/Yc6cOe0dHR2Vw4cP76qurt6/bdu2qqeeempUb++fNWvWnhUrVhy1Z88ed3R0VDzxxBNHHUoc/XUrwR391C8Kg3yWHnJaeshp6SGnpYeclh5yWnrIKQbcpZde2n7VVVdNXLZs2WtTpkzZV19fv3fixIn1xx133LtNTU17envv9OnT986ZM6e9vr7+1GOOOabztNNOe/tQYnBEHFr0AAAAAAAUqebm5q0NDQ1ls9hkc3Pz6IaGhgk9HeNWAgAAAAAAylheCwO2z7G9yfYW21/PZ98YOLa32n7R9nrbLyT7qm0/YXtz8vvoQseJ3Gwvtt1q+6WMfT3m0Gn/mozbX9r+UOEiRy45cnqz7deTsbre9nkZx76R5HST7Y8XJmrkYvt42yttb7D9su2vJPsZp0Wql5wyTouU7WG2f2G7Ocnpt5L9J9pek+TuXttHJPuHJttbkuMTCnoBeI9ecrrE9q8zxmljsp/PXpSNvBUGbFdK+p6kcyVNlnSZ7cn56h8D7uyIaMx4XMvXJf0sIk6S9LNkG4PXEknnZO3LlcNzJZ2U/Fwj6QcDFCMOzhK9N6eS9N1krDZGxApJSj57L5V0avKe7yef0Rg8UpK+FhGTJZ0h6UtJ3hinxStXTiXGabF6R9KsiGiQ1CjpHNtnSPpHpXM6SVKHpKuT9ldL6kj2fzdph8ElV04l6fqMcbo+2cdnL8pGPmcMTJW0JSJei4h3Jf1U0gV57B+FdYGkpcnrpZI+VbhQ0JeIWC2pPWt3rhxeIOnfI+3nko6yfdyABIoDliOnuVwg6acR8U5E/FrSFqU/ozFIRMSOiFiXvH5L0iuSxotxWrR6yWkujNNBLhlv3Yt+DUl+QtIsSfcn+7PHaff4vV/Sx2x7YKLFgeglp7nw2Vv6urq6uspinCbX2ZXreD4LA+MlbcvY3q7e/4eIwSskPW57re1rkn1jImJH8rpF0pjChIbDkCuHjN3i9uVkeuPijFt8yGkRSaYbT5G0RozTkpCVU4lxWrRsV9peL6lV0hOSfiVpV0SkkiaZeftDTpPjb0o6ZkADRp+ycxoR3eP028k4/a7tock+xmnpe6mtrW1UqRcHurq63NbWNkrSS7na9NfjClHcpkfE67aPlfSE7Y2ZByMibPM4iyJGDkvGDyTdonQx7xZJ/yzp8wWNCAfF9khJyyX9TUTszvzjIuO0OPWQU8ZpEYuI/ZIabR8l6UFJpxQ2Ihyu7Jzarpf0DaWLsUco/bjCBZIWFixIDJhUKjW/paXlrpaWlnqV9sL8XZJeSqVS83M1yGdh4HVJx2ds1yb7UGQi4vXkd6vtB5We2viG7eMiYkcyhaq1oEHiUOTKIWO3SEXEG92vbd8p6dFkk5wWAdtDlP4C+eOIeCDZzTgtYj3llHFaGiJil+2Vks5Uejp5VTIrIDNv3TndbrtK0ihJvytIwOhTRk7PiYjvJLvfsf0jSX+bbDNOS1xTU1OrpE8WOo7BIJ9VkeclnZSs1HqE0gvqPJzH/jEAbP+J7SO7X0uarfSUk4clfSZp9hlJ/1mYCHEYcuXwYUlXJSvvniHpzYypzBjEsu5znKP/nx72sKRLkxWyT1R60aRfDHR8yC257/huSa9ExL9kHGKcFqlcOWWcFi/bNclflWV7uKS/UHrtiJWSLkqaZY/T7vF7kaQnI4JZP4NIjpxu7B6nyTj+lP54nPLZi7KQtxkDEZGy/WVJj0mqlLQ4Il7OV/8YMGOUnlYlpf99/CQi/tv285Lus321pN9IuriAMaIPtpdJmilptO3tkm6SdKt6zuEKSecpvfDVXkmfG/CA0accOZ2ZPFIpJG2V9EVJioiXbd8naYPSK6V/KZk6icHjzyRdKenF5F5XSfo7MU6LWa6cXsY4LVrHSVqaPC2iQtJ9EfGo7Q2Sfmp7kaT/VbogpOT3f9jeovRisZcWImj0KldOn7RdI8mS1kv6q6Q9n70oG6aQCQAAAABA+SrlBRYAAAAAAEAfKAwAAAAAAFDGKAwAAAAAAFDGKAwAAAAAAFDGKAwAAAAAAFDGKAwAAAAAAFDGKAwAAAAAAFDGKAwAAAAAAFDG/g96MOXxSZ3HCwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x36 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "((#306) [0,1,2,3,4,5,6,7,8,9...],\n",
       " (#76) [306,307,308,309,310,311,312,313,314,315...])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits = TimeSplitter()(metadataset)\n",
    "splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<__main__.TSMetaDataset at 0x7fd7ffe75d30>,\n",
       " <__main__.TSMetaDataset at 0x7fd7ffe75c18>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadatasets = TSMetaDatasets(metadataset, splits=splits)\n",
    "metadatasets.train, metadatasets.valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[8.9708e-01, 2.8598e-01, 9.0524e-01,  ..., 4.7881e-01,\n",
       "           6.9086e-01, 9.7953e-01],\n",
       "          [3.9702e-01, 2.8280e-01, 7.1657e-01,  ..., 1.7420e-01,\n",
       "           1.9575e-03, 2.7200e-01],\n",
       "          [4.9516e-01, 9.2424e-01, 6.4480e-01,  ..., 8.6884e-01,\n",
       "           1.9167e-01, 3.8663e-01],\n",
       "          [3.0259e-01, 2.1004e-01, 6.3733e-01,  ..., 7.8205e-02,\n",
       "           1.5396e-01, 3.9986e-01],\n",
       "          [5.1964e-01, 3.4127e-01, 6.4531e-01,  ..., 7.1806e-02,\n",
       "           7.4778e-01, 4.2946e-01]],\n",
       " \n",
       "         [[7.5882e-01, 8.0031e-01, 7.3100e-01,  ..., 1.3822e-02,\n",
       "           8.3882e-02, 1.7649e-01],\n",
       "          [7.8212e-01, 8.4554e-01, 5.3522e-01,  ..., 8.4573e-01,\n",
       "           2.9283e-01, 4.1084e-01],\n",
       "          [7.1707e-01, 1.0961e-01, 9.9014e-01,  ..., 2.9253e-01,\n",
       "           3.3794e-01, 2.3092e-01],\n",
       "          [9.7081e-01, 9.3648e-01, 3.8191e-01,  ..., 2.8765e-01,\n",
       "           9.0285e-01, 4.7684e-01],\n",
       "          [3.2324e-01, 3.4674e-01, 8.8366e-01,  ..., 8.3131e-01,\n",
       "           1.9483e-01, 6.3751e-02]],\n",
       " \n",
       "         [[3.6577e-01, 5.3525e-01, 4.1795e-01,  ..., 3.5981e-01,\n",
       "           9.3276e-01, 7.0333e-01],\n",
       "          [6.7278e-01, 7.0413e-02, 5.7374e-01,  ..., 9.0295e-01,\n",
       "           3.6350e-01, 9.6660e-01],\n",
       "          [7.4306e-01, 8.0161e-01, 4.6418e-01,  ..., 6.9928e-01,\n",
       "           3.8255e-01, 2.8446e-01],\n",
       "          [1.1848e-01, 2.9266e-01, 4.7914e-01,  ..., 9.2846e-01,\n",
       "           9.1835e-01, 1.3424e-01],\n",
       "          [5.2314e-01, 7.8462e-01, 4.0047e-01,  ..., 2.8954e-01,\n",
       "           7.8985e-02, 5.9372e-01]],\n",
       " \n",
       "         ...,\n",
       " \n",
       "         [[3.9528e-01, 6.2661e-01, 5.0106e-01,  ..., 5.9371e-01,\n",
       "           9.4917e-01, 4.4450e-01],\n",
       "          [8.5632e-01, 5.2220e-01, 5.2169e-01,  ..., 3.6134e-01,\n",
       "           8.3527e-01, 6.9476e-01],\n",
       "          [2.6391e-01, 6.8925e-01, 8.1441e-01,  ..., 5.8711e-01,\n",
       "           2.4186e-01, 1.3854e-01],\n",
       "          [6.9608e-01, 5.8143e-01, 6.7683e-01,  ..., 3.6198e-01,\n",
       "           7.9069e-01, 2.3458e-01],\n",
       "          [9.1666e-01, 8.4379e-01, 9.7085e-01,  ..., 1.3755e-02,\n",
       "           3.3765e-02, 1.0020e-01]],\n",
       " \n",
       "         [[8.4602e-01, 3.5836e-01, 5.5184e-01,  ..., 7.9122e-01,\n",
       "           3.3502e-01, 3.9309e-01],\n",
       "          [6.2136e-01, 7.2072e-01, 7.8639e-01,  ..., 1.8939e-01,\n",
       "           3.6156e-04, 6.2199e-02],\n",
       "          [5.7941e-01, 6.6271e-01, 3.4343e-01,  ..., 7.1136e-01,\n",
       "           7.4348e-01, 6.5310e-01],\n",
       "          [1.0420e-01, 7.0913e-01, 8.8308e-01,  ..., 8.2808e-01,\n",
       "           3.4749e-01, 1.6145e-01],\n",
       "          [8.0476e-01, 1.0886e-01, 6.2308e-02,  ..., 2.7693e-02,\n",
       "           1.3562e-01, 1.8487e-01]],\n",
       " \n",
       "         [[4.3464e-01, 9.6710e-01, 3.7880e-01,  ..., 1.1528e-01,\n",
       "           5.5569e-01, 8.5616e-01],\n",
       "          [7.8498e-01, 3.6707e-01, 9.2552e-01,  ..., 8.8065e-01,\n",
       "           7.7275e-01, 9.5932e-02],\n",
       "          [5.8527e-01, 7.6148e-01, 6.1508e-01,  ..., 3.9530e-01,\n",
       "           1.9376e-01, 7.3949e-01],\n",
       "          [4.8249e-01, 8.3423e-01, 4.7482e-01,  ..., 2.5656e-01,\n",
       "           7.5617e-01, 6.9391e-01],\n",
       "          [9.5702e-01, 7.9796e-01, 5.0623e-01,  ..., 7.0712e-01,\n",
       "           4.8639e-01, 3.4118e-01]]]),\n",
       " TensorCategory([ 9,  8,  4,  3,  4, 10, 10,  4,  7,  6,  4, 10,  2, 10,  6,  2,  8,  7,\n",
       "          2,  3,  2,  3,  6,  2,  4,  9,  4,  5,  8,  2,  8,  2,  9,  1,  1, 10,\n",
       "          8,  8, 10,  6,  8,  4,  2,  2,  7,  4,  6,  1, 10,  1,  7,  4,  8,  1,\n",
       "          9,  1,  9,  1,  6,  3, 10,  8,  8,  8]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dls = TSDataLoaders.from_dsets(metadatasets.train, metadatasets.valid)\n",
    "xb, yb = first(dls.train)\n",
    "xb, yb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There also en easy way to map any particular sample in a batch to the original dataset and id: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = TSDataLoaders.from_dsets(metadatasets.train, metadatasets.valid)\n",
    "xb, yb = first(dls.train)\n",
    "mappings = dls.train.dataset.mapping_idxs\n",
    "for i, (xbi, ybi) in enumerate(zip(xb, yb)):\n",
    "    ds, idx = mappings[i]\n",
    "    test_close(dsets[ds][idx][0].data.cpu(), xbi.cpu())\n",
    "    test_close(dsets[ds][idx][1].data.cpu(), ybi.cpu())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example the 3rd sample in this batch would be: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0, 102], dtype=int32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dls.train.dataset.mapping_idxs[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "out = create_scripts(); beep(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
